/*
    Copyright 2010 Barnet Wagman

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
 */

package com.norbl.util.s3;

import com.norbl.util.*;
import java.io.*;
import org.jets3t.service.*;
import org.jets3t.service.security.*;
import org.jets3t.service.impl.rest.httpclient.*;
import org.jets3t.service.model.*;
import org.jets3t.service.utils.*;
import org.jets3t.apps.synchronize.*;
import java.util.*;

/**
 * 
 * @author Barnet Wagman
 */
public class S3Ops {

        // S3 Error codes; NOT a complete list
    static final String S3_ERROR_CODE_NO_SUCH_BUCKET = "NoSuchBucket";
    static final String S3_ERROR_CODE_NO_SUCH_KEY = "NoSuchKey";
    static final String S3_ERROR_BUCKET_ALREADY_EXISTS = "BucketAlreadyExists";
    static final String S3_ERROR_ENTITY_TOO_LARGE = "EntityTooLarge";

    static long S3_ERROR_NAP_TIME = 2L * 1000L;

    public static final String GZSER = "gzser";
    public static final String JAR = "jar";

    private String awsAccessKeyID;
    private String awsSecretKey;

    private AWSCredentials acs;
    private S3Service s3Read, s3Write;

    Jets3tProperties jp;

    private Serializer standardSerializer;
    private Serializer gzipSerializer;

    long retryIntervalMillis;
    long maxWaitMillis;

    public S3Ops(String awsAccessKeyID,String awsSecretKey,
                 long retryIntervalMillis,long maxWaitMillis)
        throws S3ServiceException {

        this.awsAccessKeyID = awsAccessKeyID;
        this.awsSecretKey = awsSecretKey;

        this.retryIntervalMillis = retryIntervalMillis;
        this.maxWaitMillis = maxWaitMillis;

        acs = new AWSCredentials(awsAccessKeyID,awsSecretKey);
        jp = createDefaultJets3tProperties();

        s3Read  = new RestS3Service(acs, null,null,jp);
        s3Write = new RestS3Service(acs, null,null,jp);

        standardSerializer = new StandardSerializer();
        gzipSerializer = new GzipSerializer();
    }

    public S3Ops cleanClone() {
        try {
            return( new S3Ops(awsAccessKeyID,awsSecretKey,
                              retryIntervalMillis,maxWaitMillis) );
        }
        catch(Throwable t) { throw new RuntimeException(t); }
    }

        /** Closes both the read and write S3 connections.  After this
         *  method has been called, you will not be able to use this
         *  object until {@link #replaceS3Read()} and/or {@link #replaceS3Write()}
         *  have been called.
         * 
         * @throws S3AccessException
         */
    public void shutdown() throws S3AccessException {
        try {
            if ( (s3Read != null) && !s3Read.isShutdown() ) s3Read.shutdown();
            if ( (s3Write != null) && !s3Write.isShutdown() ) s3Write.shutdown();
        }
        catch(S3ServiceException sx) { throw new S3AccessException(sx); }
    }

    AWSCredentials getACS() { return(acs); }
    long getRetryIntervalMillis() { return(retryIntervalMillis); }
    long getMaxWaitMillis() { return(maxWaitMillis); }
    Jets3tProperties getJet3tProperties() { return(jp); }

        // ---------- Public ops ---------------------

    public void setRetryInterval(long retryIntervalMillis) {
        this.retryIntervalMillis = retryIntervalMillis;
    }

    public void setMaxWait(long maxWaitMillis) {
        this.maxWaitMillis = maxWaitMillis;
    }

        /**
         *
         * @return the names of all buckets owned by the user.
         * @throws S3ServiceException
         */
    public List<String> listBuckets() throws S3ServiceException {
        S3Bucket[] bs = s3Read.listAllBuckets();
        List<String> bns = new ArrayList<String>();
        for ( S3Bucket b : bs ) {
            bns.add(b.getName());
        }
        return(bns);
    }

        /** <i>Warning</i>: S3 does not allow us to determine if a bucket
         *  name is already used.  This method performs a 'bucket accessible'
         *  test, which returns <tt>true</tt> if the bucket exists and we
         *  have permission to access it.  It returns <tt>false</tt> either if
         *  the bucket name does not exist or if we do not have permission
         *  so access it.  To determine if a bucket names is available, we
         *  must use <tt>createBucket()</tt>.
         * @param bucketName
         * @return true if the bucket exists, else false.
         * @throws org.jets3t.service.S3ServiceException
         */
    public boolean bucketExists(String bucketName) throws S3ServiceException {
        return( s3Read.isBucketAccessible(bucketName) );
    }

        /** 
         *
         * @param bucketName
         * @return true if successful, false if the bucket already exists.
         * @throws org.jets3t.service.S3ServiceException
         */
    public boolean createBucket(String bucketName) throws S3ServiceException {
        try {
            if ( bucketExists(bucketName) ) return(false);
            s3Write.createBucket(bucketName);
            return(true);
        }
        catch(S3ServiceException sx) {
            String errCode = sx.getS3ErrorCode();
            if ( (errCode != null) &&
                 errCode.equals(S3_ERROR_BUCKET_ALREADY_EXISTS) ) {
                return(false);
            }
            else throw sx;
        }
    }

    public boolean objectExists(String bucketName,String s3Key,
                                boolean retry)
        throws S3ServiceException {
        if (retry) return(objectExists(bucketName, s3Key,
                          retryIntervalMillis, maxWaitMillis));
        else return(objExists(bucketName, s3Key, false));
    }

    private boolean objExists(String bucketName,String s3Key,
                                boolean replaceS3Service)
        throws S3ServiceException {

        try {
            if ( replaceS3Service ) replaceS3Read();
            S3Object o = s3Read.getObjectDetails(new S3Bucket(bucketName), s3Key);
            return(true);
        }
        catch(S3ServiceException sx) {            
            if ( sx.getResponseCode() == 404 ) return(false);            
            else throw sx;            
        }
    }

    private boolean objectExists(String bucketName,String s3Key,
                                long retryIntervalMillis,
                                long maxWaitMillis)
        throws S3ServiceException {

        long t0 = System.currentTimeMillis();
        S3ServiceException s3x;
        boolean replaceS3Service = false;
        do {
            try {
                return( objectExists(bucketName, s3Key, replaceS3Service) );
            }
            catch(S3ServiceException x) {
                s3x = x;
                replaceS3Service = true;
                Time.nap(retryIntervalMillis);
            }
        }
        while ( !Time.timedOut(t0,maxWaitMillis) );
        throw s3x;
    }

       /**  Deletes the specified object from S3.
         *
         * @param bucketName
         * @param s3Key
         * 
         */
    public void deleteObject(String bucketName,String s3Key,
                             boolean retry)
        throws S3ServiceException {
        if (retry) deleteObject(bucketName, s3Key,
                                retryIntervalMillis, maxWaitMillis);
        else deleteObj(bucketName, s3Key, false);
    }


    private void deleteObj(String bucketName,String s3Key,
                             boolean replaceS3Service)
        throws S3ServiceException {
       
        if ( replaceS3Service ) replaceS3Write();
        s3Write.deleteObject(bucketName, s3Key);
    }

    private void deleteObject(String bucketName,String s3Key,
                            long retryIntervalMillis,
                            long maxWaitMillis)
        throws S3AccessException {

        long t0 = System.currentTimeMillis();
        S3AccessException s3x = null;
        boolean replaceS3Service = false;
        do {
            try {
                deleteObject(bucketName, s3Key, replaceS3Service);
                return;
            }
            catch(S3ServiceException x) {
                s3x = new S3AccessException(x);
                replaceS3Service = true;
                Time.nap(retryIntervalMillis);
            }
        }
        while ( !Time.timedOut(t0,maxWaitMillis) );
        throw s3x;
    }

        /** Retrieves an object from S3.  The object is assumed to be a
         *  <tt>byte[]</tt>.  If the <tt>s3Key</tt> ends with
         *  <tt>.gzser</tt>, the object is assumed to have been compressed
         *  with gzip.  In that case, the retrieved <tt>byte[]</tt> is first
         *  decompressed and then deserialized.
         *
         * @param bucketName
         * @param s3Key
         * @return the specified object or <tt>null</tt>.
         */
    public Object getObject(String bucketName,String s3Key,
                          boolean retry)
        throws InvalidClassException, IOException, ClassNotFoundException,
               S3ServiceException {
        if (retry) return(getObject(bucketName, s3Key,
                                    retryIntervalMillis, maxWaitMillis));
        else return(getObj(bucketName, s3Key,false));
    }

    private Object getObj(String bucketName,String s3Key,
                          boolean replaceS3Service)
        throws InvalidClassException, IOException, ClassNotFoundException,
               S3ServiceException {

        if ( replaceS3Service ) replaceS3Read();
        Serializer serializer = getSerializer(s3Key);
        return(
            serializer.byteArrayToObject(
                getBytes(s3Read.getObject(new S3Bucket(bucketName), s3Key)) ) );
    }

    private Object getObject(String bucketName,String s3Key,
                            long retryIntervalMillis,
                            long maxWaitMillis)
        throws InvalidClassException, IOException, ClassNotFoundException,
               S3ServiceException {

        long t0 = System.currentTimeMillis();
        S3ServiceException s3x;
        boolean replaceS3Service = false;
        do {
            try {
                return( getObj(bucketName, s3Key,replaceS3Service) );
            }
            catch(S3ServiceException x) {
                s3x = x;
                replaceS3Service = true;
                Time.nap(retryIntervalMillis);
            }
        }
        while ( !Time.timedOut(t0,maxWaitMillis) );

        throw s3x;
     }
    
        /** Adds the <tt>object</tt> to S3, per the <tt>bucketName</tt> and
         *  <tt>s3Key</tt>.  If the <tt>s3Key</tt> ends with <tt>.gzser</tt>,
         *      the object will be compressed using gzip.<p>
         *
         *  Details: the object is first serialized to a <tt>byte[]</tt>.
         *  If the <tt>s3Key</tt> does not ends with <tt>.gzser</tt>,
         *  this <tt>byte[]</tt> is written to S3.  If the key does end
         *  with <tt>.gzser</tt>, the <tt>byte[]</tt> is compressed using
         *  gzip (see <tt>java.util.gzip</tt>) and the compressed <tt>byte[]</tt>
         *  is written to S3.
         *
         * @param bucketName
         * @param s3Key If the <tt>s3KeyName</tt> ends with <tt>.gzser</tt>,
         *      the object will be compressed using
         *      <tt>java.util.zip.GZIPInputStream</tt> before it is written
         *      to S3.  Otherwise it is written without compression.
         * @param object
         */
    public void addObject(String bucketName,String s3Key,
                          Serializable object,
                          boolean retry)
        throws InvalidClassException, IOException, ClassNotFoundException,
               S3ServiceException {
        if (retry) addObject(bucketName, s3Key, object,
                             retryIntervalMillis, maxWaitMillis);
        else addObj(bucketName, s3Key, object, false);
    }

    private void addObj(String bucketName,String s3Key,
                          Serializable object,
                          boolean replaceS3Service)
        throws IOException, S3ServiceException {

        if ( replaceS3Service ) replaceS3Write();
        Serializer serializer = getSerializer(s3Key);        
        putBytes(bucketName,s3Key,serializer.objectToByteArray(object));        
    }

     private void addObject(String bucketName,String s3Key,
                            Serializable object,
                            long retryIntervalMillis,
                            long maxWaitMillis)
        throws InvalidClassException, IOException, ClassNotFoundException,
               S3ServiceException {

        long t0 = System.currentTimeMillis();
        S3ServiceException s3x;
        boolean replaceS3Service = false;
        do {
            try {
                addObj(bucketName, s3Key,object,replaceS3Service);                
                return;
            }
            catch(S3ServiceException x) {               
                s3x = x;
                replaceS3Service = true;
                Time.nap(retryIntervalMillis);
            }
        }
        while ( !Time.timedOut(t0,maxWaitMillis) );

        throw s3x;
     }

        /** This method blocks until all objects exist in S3,
         *  the max time has elapsed, or an exception is thrown.
         *
         * @param soks
         * @param maxWaitTime maximum time (in milliseconds) that
         *  this method will block waiting for objects to become
         *  available in S3.
         *
         * @return the list of <tt>Throwable</tt>s. If the add
         *  operations were all successful, and empty list is
         *  returned.
         *
         * @throws S3AccessException if the max time has elapsed
         *  before all objects are available in S3 or an
         *  error occurred.
         */
     public List<Throwable> add(List<S3Bko> soks, long maxWaitTime)
        throws S3AccessException {

         ObjectAdder oa = ObjectAdder.add(this, soks);

         long t0 = System.currentTimeMillis();

         while ( (System.currentTimeMillis() - t0) < maxWaitTime ) {
             if ( oa.isDone() || oa.hasThrowables() )
                 return(oa.getThrowables());
         }

         String mess = "The max time (" +
                maxWaitTime + " millis) expired before all " +
                " objects were accessible in S3. ";
         if ( oa.hasThrowables() ) {
             List<Throwable> xs = oa.getThrowables();
             mess += " N throwables=" + xs.size() +
                " Last throwable: " +
                StringUtil.throwableToString(xs.get(xs.size()-1));
         }
         throw new S3AccessException(mess);
     }

        /** Copies jars from S3 to a local dir. Jars are identified by
         *  object name extension {@link #JAR}. Jars objects are
         *  copied literally (i.e. no decompression). Note that all
         *  jars with the prefix <tt>s3KeyPrefix</tt> will be copied,
         *  even those in pseudo-subdirectories.  The jars are written to
         *  a single directory (i.e. any directory structure is lost). If
         *  <tt>dir</tt> does not exist, it will be created.<P>
         *
         *  Warning: this method will overwrite files.
         *           
         * @param dir Fully specified pathname of a local directory.
         * @param bucketName
         * @param s3KeyPrefix
         * @return the number of jars copied.
         */
    public int getJarsFromS3(File dir, String bucketName, String s3KeyPrefix,
                             boolean verbose,boolean retry)
        throws FileNotFoundException, IOException, S3ServiceException {
       
        if ( !dir.exists()  ) dir.mkdirs();
        if ( !dir.isDirectory() ) throw new NotADirectoryException(
            dir.getPath() + " is a file, not a directory.");

        S3Bucket bucket = new S3Bucket(bucketName);

        S3Object[] sos = listObjects(bucket, s3KeyPrefix, retry);
                // s3Read.listObjects(bucket, s3KeyPrefix,null);
        if ( verbose ) System.out.println("N files found in " +
                s3KeyPrefix + "=" + sos.length);

        if ( (sos == null) || (sos.length < 1) ) return(0);

        int nCopied = 0;

        NEXT: for (S3Object so : sos ) {
            String key = so.getKey();
            if ( !key.endsWith("." + JAR) ) continue NEXT;
            String oName = getObjectName(key);
            if ( oName == null ) continue NEXT;
            File f = new File(dir,oName);
            S3Object sob = s3Read.getObject(bucket,so.getKey());
                // ^ This second copy of the S3Object is necessary.
                //   The S3Object obtained using listObjects()
                //   appears to have a closed stream.
            byte[] bytes = getBytes(sob);
            if ( f.exists() ) f.delete();
            writeToFile(f,bytes);
            ++nCopied;
            if ( verbose ) System.out.println("Got from S3: " + f.getPath());
        }

        return(nCopied);        
    }

    public int getJarsFromS3(File dir, String bucketName, String s3KeyPrefix,
                             boolean retry)
        throws FileNotFoundException, IOException, S3ServiceException {
        return( getJarsFromS3(dir, bucketName, s3KeyPrefix,false,retry) );
    }

        /**
         * <p>
         * This method uses <tt>org.jets3tapps.synchronize.Synchronize</tt>.
         *
         * @param jarBucketName
         * @param jarS3KeyPrefix
         * @param jarDir
         */
    public void uploadJars(String jarBucketName, String jarS3KeyPrefix,
                           File jarDir) {
        try {
            if ( !jarDir.exists() || !jarDir.isDirectory() )
                throw new RuntimeException(jarDir.getPath() +
                    " does not exist or is not a dir.");

                // jets3t sync wants a prefix with the bucket name
                // prepended, with no leading '/'.
            String s3DirPath = jarBucketName + "/" + jarS3KeyPrefix;

                // Get the local paths of jars.
            List<File> jars = new ArrayList<File>();
            File[] fs = jarDir.listFiles();
           
            if ( (fs == null) || (fs.length < 1) ) return;
            for ( File f : fs ) {
                if ( !f.isDirectory() &&
                     f.getPath().toLowerCase().endsWith(".jar") ) {
                    jars.add(f);
                }
            }
           
            if ( jars.size() < 1 ) return;

                // Specify the sync action.
            Synchronize s =
                new Synchronize(s3Write,
                                true, // doAction,
                                true, // isQuiet,
                                true, // isNoProgress,
                                false, // isForce: false prevents writing if unchanged.
                                false, // isKeepFiles,
                                false, // isNoDelete,
                                false, // isMoveEnabled,
                                false, // isBatchMode,
                                false, // isSkipMetadata,
                                false, // isGzipEnabled,
                                false, // isEncryptionEnabled,
                                0, // reportLevel: 0=summary only
                                jp);

             s.run(s3DirPath,
                     jars,"" +
                     "UP",null,
                     "PRIVATE" // acl string
                     );
        }
        catch(Exception x) { throw new S3AccessException(x); }
    }

    S3Service getWriteService() { return(s3Write); }
    S3Service getReadService() { return(s3Read); }

    private S3Object[] listObjects(S3Bucket bucket, String s3KeyPrefix,
                                   boolean retry)
        throws S3ServiceException {
        if (retry) return(listObjectsRetry(bucket,s3KeyPrefix));
        else return(listObjectsNoRetry(bucket,s3KeyPrefix,false));
    }

    public List<String> listAllObjects(String bucketName,String s3KeyPrefix,
                                       boolean retry)
        throws S3ServiceException {

        S3Bucket bucket = new S3Bucket(bucketName);
        S3Object[] o3s = listObjects(bucket, s3KeyPrefix, retry);
        List<String> keys = new ArrayList<String>();
        for ( S3Object o : o3s ) {
            keys.add(o.getKey());
        }
        return(keys);
    }

    private S3Object[] listObjectsNoRetry(S3Bucket bucket, String s3KeyPrefix,
                                         boolean replaceService)
        throws S3ServiceException {
     
        if (replaceService) replaceS3Read();
        return(s3Read.listObjects(bucket, s3KeyPrefix,null));
    }

    private S3Object[] listObjectsRetry(S3Bucket bucket, String s3KeyPrefix)
        throws S3ServiceException {
        long t0 = System.currentTimeMillis();
        S3ServiceException s3x;
        boolean replaceS3Service = false;
        do {
            try {
                return(listObjectsNoRetry(bucket,s3KeyPrefix,replaceS3Service));
            }
            catch(S3ServiceException x) {
                s3x = x;
                replaceS3Service = true;
                Time.nap(retryIntervalMillis);
            }
        }
        while ( !Time.timedOut(t0,maxWaitMillis) );

        throw s3x;
    }


    public int deleteObjects(String bucketName, String s3KeyPrefix,
                             boolean retry)
        throws S3ServiceException {

        S3Bucket bucket = new S3Bucket(bucketName);

        S3Object[] sos = listObjects(bucket, s3KeyPrefix, retry);
                // s3Read.listObjects(bucket, s3KeyPrefix,null);

        int nDeleted = 0;
        for ( S3Object so : sos ) {
            try {                
                deleteObject(bucketName,so.getKey(),retry);
                ++nDeleted;
            }
            catch(S3ServiceException sx) {
                System.out.println(StringUtil.throwableToString(sx));
            }
        }
        return(nDeleted);
    }

     public List<PropertyList> findBucketSizes(boolean retry)
        throws S3ServiceException {

        S3Bucket[] bs = s3Read.listAllBuckets();
        List<PropertyList> pls = new ArrayList<PropertyList>();
        for (S3Bucket b : bs ) {
            pls.add(findBucketSize(b,retry));
        }
        return(pls);
    }

    public PropertyList findBucketSize(S3Bucket bucket,boolean retry)
        throws S3ServiceException {

        S3Object[] o3s = listObjects(bucket, null, retry);
        
        long nBytes = 0L;
        
        for ( S3Object s3o : o3s ) {
            nBytes += s3o.getContentLength();
        }

        PropertyList pl = new PropertyList();
        pl.add("bucketName", bucket.getName());
        pl.add("N objects",o3s.length);
        pl.add("N bytes",new Long(nBytes));
        return(pl);
    }

    public int dummy() { return(1); }

        // -------------------------------------------------------------

    private void replaceS3Read() throws S3ServiceException {
        if ( (s3Read != null) && !s3Read.isShutdown() ) s3Read.shutdown();
        s3Read = new RestS3Service(acs, null,null,jp);
    }

    private void replaceS3Write() throws S3ServiceException {
        if ( (s3Write != null) && !s3Write.isShutdown() ) s3Write.shutdown();
        s3Write = new RestS3Service(acs, null,null,jp);
    }

    Serializer getSerializer(String s3Key) {
        if ( s3Key.endsWith("." + GZSER) ) return(gzipSerializer);
        else if ( s3Key.endsWith("." + JAR) ) return(standardSerializer);
        else return(standardSerializer);
    }

    private String getObjectName(String key) {

        String k = key.trim();

        int idx = k.lastIndexOf("/");
        if ( idx < 0 ) return(null);
        if ( (k.length() - 1) < (idx + 1) ) return(null);
        String nm = k.substring(idx+1);
        if ( nm.length() > 0 ) return(nm);
        else return(null);
    }

    private void writeToFile(File f, byte[] bytes) throws IOException {

        FileOutputStream fo = null;
        BufferedOutputStream buffo = null;

        try {
            fo = new FileOutputStream(f);
            buffo = new BufferedOutputStream(fo);
            buffo.write(bytes);
            buffo.flush();
            fo.flush();
        }
        finally {
            if ( buffo != null ) buffo.close();
            if ( fo != null ) fo.close();
        }
    }

    private byte[] readFromFile(File f) throws IOException {

        FileInputStream fi = null;
        BufferedInputStream buffi = null;

        try {
            fi = new FileInputStream(f);
            buffi = new BufferedInputStream(fi);
            int len = (int) f.length();
            byte[] buffer = new byte [len];
            int nRead = buffi.read(buffer);
            if ( nRead != len )
                throw new S3AccessException("File read error: file length=" +
                                len + " but only " + nRead + " were obtained.");
            return(buffer);
        }
        finally {
            if ( buffi != null ) buffi.close();
            if ( fi != null ) fi.close();
        }
    }

        /** This methods sets values for all properties in the default
         *  property file in the jet3set package except for gui related
         *  properties.
         *
         * @return
         */
    private Jets3tProperties createDefaultJets3tProperties() {

        Jets3tProperties jp = new Jets3tProperties();

        // ###
        // # Upload properties for applications.
        // ###
        jp.setProperty("uploads.storeEmptyDirectories","true");

        // ###
        // # Download properties for applications.
        // ###
        jp.setProperty("downloads.restoreLastModifiedDate","false");

        // ###
        // # S3Service properties
        // ###
        jp.setProperty("s3service.https-only","true"); // true
        jp.setProperty("s3service.max-thread-count","8"); // 4
        jp.setProperty("s3service.admin-max-thread-count","10");
        jp.setProperty("s3service.stream-retry-buffer-size","524288"); // "131072");
        jp.setProperty("s3service.internal-error-retry-max","5");
        jp.setProperty("s3service.default-bucket-location","US");
//        jp.setProperty("s3service.s3-endpoint","INCORRECT"); // "s3.amazonaws.com"); // Set to "incorrect" for test.
        jp.setProperty("s3service.disable-dns-buckets","false");

        // #s3service.ignore-exceptions-in-multi=false

        // ###
        // # REST/HTTP HttpClient Implementation properties
        // ###
        jp.setProperty("httpclient.connection-timeout-ms","60000");
        jp.setProperty("httpclient.socket-timeout-ms",
                        "300000"// "60000"
                        );
        jp.setProperty("httpclient.max-connections", "10" );// "10");
        jp.setProperty("httpclient.stale-checking-enabled","true");
        jp.setProperty("httpclient.retry-max","5");
        // #httpclient.useragent=My User Agent String
        // #httpclient.read-throttle=50
        jp.setProperty("httpclient.authentication-preemptive","false");
        jp.setProperty("httpclient.proxy-autodetect","true");
        // #httpclient.proxy-host=
        // #httpclient.proxy-port=

        // # TCP window size hints for kernel
        // #httpclient.socket-receive-buffer=65536
        // #httpclient.socket-send-buffer=65536

        // ###
        // # File/Object comparison properties
        // ###
        // #filecomparer.use-md5-files=false
        // #filecomparer.generate-md5-files=false
        // #filecomparer.skip-upload-of-md5-files=false
        // #filecomparer.ignore-panic-dir-placeholders=false
        // #filecomparer.assume-local-latest-in-mismatch=false

        // ###
        // # XML Parsing properties
        // ###
        jp.setProperty("xmlparser.sanitize-listings","true");

        // ###
        // # Encryption Algorithm
        // #
        // # Note that some cryptography options will only be available
        // # if the Java Cryptography Extension (JCE) Unlimited Strength
        // # Jurisdiction Policy Files are installed.
        // ###
        jp.setProperty("crypto.algorithm","PBEWithMD5AndDES");

        return(jp);
    }

    private void putBytes(String bucketName,String key,byte[] b)
        throws S3ServiceException, IOException {

        S3Object o = null;
        try {
            o = new S3Object(key);
            o.setDataInputStream(new ByteArrayInputStream(b));
            o.setContentLength(b.length);
            o.setContentType(Mimetypes.MIMETYPE_OCTET_STREAM); // added 3/2/2010
            s3Write.putObject(bucketName,o);  
        }       
        finally {
            if ( o != null ) { o.closeDataInputStream(); o = null; }
        }
    }

    private byte[] getBytes(S3Object o3) throws IOException, S3ServiceException {

        BufferedInputStream ins = null;
        try {
            ins = new BufferedInputStream(o3.getDataInputStream());
            return( readBytes(ins,(int) o3.getContentLength()) );
        }

        finally {
            if ( o3 != null ) { o3.closeDataInputStream(); o3 = null; }
            if ( ins != null ) { ins.close(); ins = null; }
        }
    }

    private byte[] readBytes(BufferedInputStream ins, int len)
        throws IOException {

        byte[] bytes = new byte [len];
        int nRead = 0;
        while ( nRead < len ) {            
            int n = ins.read(bytes, nRead, len - nRead);
            if ( n > 0 ) nRead += n;
        }
        return(bytes);
    }
}
